package download

import (
	"bytes"
	"cd-crawler/tools"
	"io"
	"io/ioutil"
	"log"
	"net/http"
)

func DownUrl(url string) (io.Reader, error) {
	//client := &http.Client{CheckRedirect: func(
	//	req *http.Request,
	//	via []*http.Request) error {
	//	fmt.Println("Redirect:", req)
	//	return nil
	//}}

	//req, err := http.NewRequest("GET", url, nil)
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	//req.Header.Add("User-Agent", `Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3741.400 QQBrowser/10.5.3863.400`)
	//resp, err := client.Do(req)
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		log.Printf("get url(%s) status code error: %d %s", url, resp.StatusCode, resp.Status)
	}

	reader, err := tools.DecodeHTMLBody(resp.Body, "")
	b, _ := ioutil.ReadAll(reader)
	bodyReader := bytes.NewReader(b)

	return bodyReader, nil

	//reader, err := tools.DecodeHTMLBody(resp.Body, "")
	//bytes, err := ioutil.ReadAll(reader)
	//fmt.Println(string(bytes))

	//return "", nil

}
